{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "using ReinforcementLearningAnIntroduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Distributions\n",
    "using Plots\n",
    "\n",
    "const pₕ = 0.4\n",
    "const WinCapital = 100\n",
    "\n",
    "decode_state(s::Int) = s - 1\n",
    "encode_state(s::Int) = s + 1\n",
    "\n",
    "function nextstep(s::Int, a::Int)\n",
    "    s = decode_state(s)\n",
    "    a = min(s, a)\n",
    "    if s == WinCapital\n",
    "        [(nextstate=encode_state(s), reward=0., prob=1.0)]\n",
    "    elseif s == 0\n",
    "        [(nextstate=encode_state(s), reward=0., prob=1.0)]\n",
    "    else\n",
    "        [(nextstate=encode_state(min(s+a, WinCapital)), reward= s+a >= WinCapital ? 1.0 : 0., prob=pₕ),\n",
    "         (nextstate=encode_state(max(s-a, 0)), reward=0., prob=1-pₕ)]\n",
    "    end\n",
    "end\n",
    "\n",
    "const GamblerProblemEnvModel = DeterministicDistributionModel([nextstep(s, a) for s in 1:(WinCapital+1), a in 1:WinCapital]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip6800\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip6800)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip6801\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip6800)\" d=\"\n",
       "M153.898 1487.47 L2352.76 1487.47 L2352.76 47.2441 L153.898 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip6802\">\n",
       "    <rect x=\"153\" y=\"47\" width=\"2200\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  194.963,1487.47 194.963,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  724.145,1487.47 724.145,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1253.33,1487.47 1253.33,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1782.51,1487.47 1782.51,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2311.69,1487.47 2311.69,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1449.58 2352.76,1449.58 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1167.19 2352.76,1167.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,884.807 2352.76,884.807 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,602.42 2352.76,602.42 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,320.033 2352.76,320.033 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 153.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  194.963,1487.47 194.963,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  724.145,1487.47 724.145,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1253.33,1487.47 1253.33,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1782.51,1487.47 1782.51,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2311.69,1487.47 2311.69,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1449.58 180.284,1449.58 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1167.19 180.284,1167.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,884.807 180.284,884.807 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,602.42 180.284,602.42 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,320.033 180.284,320.033 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 194.963, 1541.47)\" x=\"194.963\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 724.145, 1541.47)\" x=\"724.145\" y=\"1541.47\">25</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1253.33, 1541.47)\" x=\"1253.33\" y=\"1541.47\">50</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1782.51, 1541.47)\" x=\"1782.51\" y=\"1541.47\">75</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2311.69, 1541.47)\" x=\"2311.69\" y=\"1541.47\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1467.08)\" x=\"129.898\" y=\"1467.08\">0.0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1184.69)\" x=\"129.898\" y=\"1184.69\">0.2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 902.307)\" x=\"129.898\" y=\"902.307\">0.4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 619.92)\" x=\"129.898\" y=\"619.92\">0.6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 337.533)\" x=\"129.898\" y=\"337.533\">0.8</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip6802)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,1446.71 237.297,1442.3 258.464,1436.56 279.632,1431.36 300.799,1425.04 321.966,1417.02 343.134,1410.31 364.301,1404.01 385.468,1396.38 406.635,1388.22 \n",
       "  427.803,1378.49 448.97,1368.17 470.137,1357.5 491.305,1351.4 512.472,1344.49 533.639,1335.66 554.806,1327.3 575.974,1316.56 597.141,1303.97 618.308,1296.16 \n",
       "  639.476,1285.85 660.643,1271.85 681.81,1260.98 702.977,1246.06 724.145,1223.67 745.312,1219.3 766.479,1212.74 787.647,1204.13 808.814,1196.33 829.981,1186.85 \n",
       "  851.148,1174.83 872.316,1164.76 893.483,1155.32 914.65,1143.86 935.818,1131.62 956.985,1117.03 978.152,1101.56 999.319,1085.51 1020.49,1076.4 1041.65,1066.04 \n",
       "  1062.82,1052.78 1083.99,1040.24 1105.16,1024.14 1126.32,1005.23 1147.49,993.544 1168.66,978.066 1189.83,957.059 1210.99,940.763 1232.16,918.381 1253.33,884.807 \n",
       "  1274.49,880.44 1295.66,873.875 1316.83,865.269 1338,857.466 1359.16,847.989 1380.33,835.962 1401.5,825.9 1422.67,816.451 1443.83,804.995 1465,792.757 \n",
       "  1486.17,778.168 1507.33,762.691 1528.5,746.641 1549.67,737.539 1570.84,727.17 1592,713.917 1613.17,701.374 1634.34,685.278 1655.51,666.362 1676.67,654.679 \n",
       "  1697.84,639.202 1719.01,618.194 1740.17,601.898 1761.34,579.516 1782.51,545.943 1803.68,539.383 1824.84,529.538 1846.01,516.635 1867.18,504.929 1888.35,490.712 \n",
       "  1909.51,472.673 1930.68,457.581 1951.85,443.408 1973.01,426.225 1994.18,407.866 2015.35,385.975 2036.52,362.768 2057.68,338.688 2078.85,325.039 2100.02,309.486 \n",
       "  2121.19,289.607 2142.35,270.793 2163.52,246.643 2184.69,218.271 2205.85,200.75 2227.02,177.534 2248.19,146.021 2269.36,121.579 2290.52,88.0053 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip6800)\" d=\"\n",
       "M1989.93 251.724 L2280.76 251.724 L2280.76 130.764 L1989.93 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1989.93,251.724 2280.76,251.724 2280.76,130.764 1989.93,130.764 1989.93,251.724 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip6800)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2013.93,191.244 2157.93,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip6800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2181.93, 208.744)\" x=\"2181.93\" y=\"208.744\">y1</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V = TabularApproximator(n_state=1+WinCapital)\n",
    "value_iteration!(V=V, model=GamblerProblemEnvModel, γ=1.0, max_iter=1000)\n",
    "plot(V.table[2:end-1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.0",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
